package com.atguigu.bigdata.spark.rdd.instance;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

import java.util.Arrays;
import java.util.List;

/**
 * @Author gmd
 * @Description 从内存中创建RDD模型（分区），梳理隔断分区数据原则
 * @Date 2024-06-02 18:44:52
 */
public class Spark02_RDD_Memory_Partition_Data {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf();
        conf.setMaster("local[*]");
        conf.setAppName("spark");
        final JavaSparkContext jsc = new JavaSparkContext(conf);

        final List<Integer> names = Arrays.asList(1, 2, 3, 4, 5, 6);
        /*
          分区数据算法原理如下：
          【1】
          【2，3】
          【4】
          【5，6】
          -------------------------------
          len=6, partNum=4     6条数据，4个分区

          (0 until 4) => [0, 1, 2, 3]

          0 => ((i * length) / numSlices, (((i + 1) * length) / numSlices))
            => ((0 * 6) / 4, (((0 + 1) * 6) / 4))
            => (0, 1) => 1
          1 => ((i * length) / numSlices, (((i + 1) * length) / numSlices))
            => ((1 * 6) / 4, (((2) * 6) / 4))
            => (1, 3) => 2
          2 => ((i * length) / numSlices, (((i + 1) * length) / numSlices))
            => ((2 * 6) / 4, (((3) * 6) / 4))
            => (3, 4) => 1
          3 => ((i * length) / numSlices, (((i + 1) * length) / numSlices))
            => ((3 * 6) / 4, (((4) * 6) / 4))
            => (4, 6) => 2
         */

        // Spark分区数据的存储基本原则：平均分
        final JavaRDD<Integer> rdd = jsc.parallelize(names, 4);

        rdd.saveAsTextFile("output");


        jsc.close();

    }
}
